import json
import re
from html.parser import HTMLParser
from lxml import html
from crawler_baidu.logger import LOG

etree = html.etree


def replace_string(str):
    """
    将所有的字符串进行标准化,替换掉一些非正常数据,[1]等无关紧要的数据,便于进行存储
    :param str: 待标准化的数据
    :return: 标准化之后的数据
    """
    str_1 = str.replace(u'\u3000', u'').replace(u'\xa0', u''). \
        replace('\n', '').replace(' ', '').replace('，', ','). \
        replace('、', ',').replace(', ', ',').replace('    ', '').replace(
        '词条图册更多图册', '')
    str_1 = re.sub(r'(\[[\d+]*\])', '', str_1)
    str_1 = re.sub(r'(\([\d+]*张\))', '', str_1)
    return str_1


def get_bk_brand_name(raw_html):
    """
    获取百度百科品牌名称
    :param raw_html: 原始的html页面源码
    :return: 百度百科品牌名称
    """
    html1 = etree.HTML(raw_html)
    bk_brand_name_1 = html1.xpath(
        '//dd[@class="lemmaWgt-lemmaTitle-title J-lemma-title"]/span')[0]
    bk_brand_name_1 = html.tostring(bk_brand_name_1)
    bk_brand_name_1 = HTMLParser().unescape(
        bk_brand_name_1.decode('utf-8'))
    pattern = re.compile(r'>(.*?)<', flags=re.DOTALL)
    bk_brand_name = pattern.findall(bk_brand_name_1)
    bk_brand_name = replace_string(''.join(bk_brand_name))
    if bk_brand_name == '':
        return 'NULL'
    else:
        return bk_brand_name


def get_bk_introduce_one(raw_html):
    """
    获取百度百科对于品牌的介绍
    :param raw_html: 原始的html页面源码
    :return: 百度百科对于品牌的介绍
    """
    html1 = etree.HTML(raw_html)
    if html1.xpath('//div[@class="lemma-summary"]'):
        bk_introduce_one = html1.xpath('//div[@class="lemma-summary"]')[0]
        bk_introduce_one = html.tostring(bk_introduce_one)
        bk_introduce_one = HTMLParser().unescape(
            bk_introduce_one.decode('utf-8'))
        # pattern = re.compile(r'>(.*?|\n.*?)<')
        pattern = re.compile(r'>(.*?)<', flags=re.DOTALL)
        bk_introduce_one = pattern.findall(bk_introduce_one)
        bk_introduce_one = replace_string(''.join(bk_introduce_one))
        LOG.info('%s' % (bk_introduce_one))
        if bk_introduce_one == '':
            return 'NULL'
        else:
            return bk_introduce_one
    else:
        return 'NULL'


def get_bk_introduce_dict_one(raw_html):
    """
    获取百度百科对于品牌描述的key,value值
    :param raw_html: 原始的html页面源码
    :return: 描述信息组成的字典值
    """
    bk_introduce_dict_one = {}
    html1 = etree.HTML(raw_html)
    for temp in ['left', 'right']:
        key_list = []
        value_list = []
        if html1.xpath(
                '//dl[@class="basicInfo-block basicInfo-{}"]'.format(temp)):
            bk_introduce_one = html1.xpath(
                '//dl[@class="basicInfo-block basicInfo-{}"]'.format(temp))[0]
            for i in bk_introduce_one.xpath('dt'):
                data_key = i.xpath('text()')[0]
                data_key = replace_string(data_key)
                LOG.info('%s' % (data_key))
                key_list.append(data_key)
            for i in bk_introduce_one.xpath('dd'):
                i = html.tostring(i)
                i = HTMLParser().unescape(i.decode('utf-8'))
                pattern = re.compile(r'>(.*?)<', flags=re.DOTALL)
                i = pattern.findall(i)
                data_value = replace_string(''.join(i))
                value_list.append(data_value)
            left_dict = dict(map(lambda x, y: [x, y], key_list, value_list))
            bk_introduce_dict_one.update(left_dict)
    LOG.info('%s' % (bk_introduce_dict_one))
    if bk_introduce_dict_one == {}:
        return 'NULL'
    else:
        return bk_introduce_dict_one


def get_bk_introduce_dict_two(raw_html):
    """
    获取百度百科对于品牌描述的key,value值
    :param raw_html: 原始的html页面源码
    :return:
    """
    html1 = etree.HTML(raw_html)
    data_index = html1.xpath('//dt[@class="catalog-title level1"]')
    bk_introduce_dict_two = {}
    label_module_list = html1.xpath(
        '//div[@label-module="para"]')  # 所有存在所需要信息的节点
    for data in data_index:
        i = int(data.xpath('a/@href')[0].split('#')[1])
        class_name = html1.xpath(
            '//div[@data-index="{}"]/h2/text()'.format(i))
        LOG.info('%s' % (class_name))
        data_index_first = html1.xpath(
            '//div[@data-index="{}"]/following::*'.format(
                i))  # 选取当前节点的所有后代元素（子、孙等）以及当前节点本身。
        data_index_last = html1.xpath(
            '//div[@data-index="{}"]/preceding::*'.format(
                i + 1))  # 选取当前节点之前的所有同级节点。
        if data_index_first == []:
            # 直接跳出
            break
        if data_index_last == []:
            # 取到最后一个介绍信息
            data_index_last = html1.xpath(
                '//div[@data-index="{}"]/following::*'.format(i))
            LOG.info('%s' % (data_index_last))
            results = [i for i in label_module_list if i in data_index_last]
        else:
            # 正常情况使用两个定位元素确定之间的文字信息
            results = [i for i in data_index_first if
                       i in data_index_last and i in label_module_list]
        result_list = []
        for result in results:
            result = html.tostring(result)
            result = HTMLParser().unescape(result.decode('utf-8'))
            pattern = re.compile(r'>(.*?)<', flags=re.DOTALL)
            result = pattern.findall(result)
            result_list.extend(result)
        value_content = replace_string(''.join(result_list))
        LOG.info('%s' % (value_content))
        bk_introduce_dict_two.update({class_name[0]: value_content})
    LOG.info('%s' % (bk_introduce_dict_two))
    if bk_introduce_dict_two == {}:
        return 'NULL'
    else:
        return bk_introduce_dict_two


def xpath_detail(raw_html):
    bk_brand_name = get_bk_brand_name(raw_html)
    bk_introduce_one = get_bk_introduce_one(raw_html)
    bk_introduce_dict_one = get_bk_introduce_dict_one(raw_html)
    bk_introduce_json_one = json.dumps(bk_introduce_dict_one,
                                       ensure_ascii=False)
    bk_introduce_dict_two = get_bk_introduce_dict_two(raw_html)
    bk_introduce_json_two = json.dumps(bk_introduce_dict_two,
                                       ensure_ascii=False)
    return bk_brand_name, bk_introduce_one, bk_introduce_json_one, bk_introduce_json_two


if __name__ == '__main__':
    with open('test.html', 'r', encoding='utf8') as f:
        content = f.read()
    LOG.info('%s' % (content))
    bk_brand_name, bk_introduce_one, bk_introduce_json_one, bk_introduce_json_two = xpath_detail(
        content)
    LOG.info('%s' % (bk_brand_name))
    LOG.info('%s' % (bk_introduce_json_two))
